{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Predict wine quality"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Use the latest versions of DataFrame and KotlinDL libraries from [version repository](https://github.com/Kotlin/kotlin-jupyter-libraries)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "pycharm": {
     "is_executing": true
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "            <div id=\"kotlin_out_0\"/>\n",
       "            <script type=\"text/javascript\">\n",
       "                            if(!window.kotlinQueues) {\n",
       "                window.kotlinQueues = {};\n",
       "            }\n",
       "            if(!window.kotlinQueues[\"DataFrame\"]) {\n",
       "                var resQueue = [];\n",
       "                window.kotlinQueues[\"DataFrame\"] = resQueue;\n",
       "                window[\"call_DataFrame\"] = function(f) {\n",
       "                    resQueue.push(f);\n",
       "                }\n",
       "            }\n",
       "            (function (){\n",
       "                var modifiers = [(function(script) {\n",
       "    script.src = \"https://cdn.jsdelivr.net/gh/Kotlin/dataframe@d6467c1389bc031958acd5310fce39167b5cc581/src/main/resources/init.js\"\n",
       "    script.type = \"text/javascript\";\n",
       "})];\n",
       "                var e = document.getElementById(\"kotlin_out_0\");\n",
       "                modifiers.forEach(function (gen) {\n",
       "                    var script = document.createElement(\"script\");\n",
       "                    gen(script)\n",
       "                    script.addEventListener(\"load\", function() {\n",
       "                        window[\"call_DataFrame\"] = function(f) {f();};\n",
       "                        window.kotlinQueues[\"DataFrame\"].forEach(function(f) {f();});\n",
       "                        window.kotlinQueues[\"DataFrame\"] = [];\n",
       "                    }, false);\n",
       "                    script.addEventListener(\"error\", function() {\n",
       "                        window[\"call_DataFrame\"] = function(f) {};\n",
       "                        window.kotlinQueues[\"DataFrame\"] = [];\n",
       "                        var div = document.createElement(\"div\");\n",
       "                        div.style.color = 'darkred';\n",
       "                        div.textContent = 'Error loading resource DataFrame';\n",
       "                        document.getElementById(\"kotlin_out_0\").appendChild(div);\n",
       "                    }, false);\n",
       "                    \n",
       "                    e.appendChild(script);\n",
       "                });\n",
       "            })();\n",
       "            </script>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "                <style>\n",
       "                :root {\n",
       "    --background: #fff;\n",
       "    --background-odd: #f5f5f5;\n",
       "    --background-hover: #d9edfd;\n",
       "    --header-text-color: #474747;\n",
       "    --text-color: #848484;\n",
       "    --text-color-dark: #000;\n",
       "    --text-color-medium: #737373;\n",
       "    --text-color-pale: #b3b3b3;\n",
       "    --inner-border-color: #aaa;\n",
       "    --bold-border-color: #000;\n",
       "    --link-color: #296eaa;\n",
       "    --link-color-pale: #296eaa;\n",
       "    --link-hover: #1a466c;\n",
       "}\n",
       "\n",
       ":root[theme=\"dark\"], :root [data-jp-theme-light=\"false\"]{\n",
       "    --background: #303030;\n",
       "    --background-odd: #3c3c3c;\n",
       "    --background-hover: #464646;\n",
       "    --header-text-color: #dddddd;\n",
       "    --text-color: #b3b3b3;\n",
       "    --text-color-dark: #dddddd;\n",
       "    --text-color-medium: #b2b2b2;\n",
       "    --text-color-pale: #737373;\n",
       "    --inner-border-color: #707070;\n",
       "    --bold-border-color: #777777;\n",
       "    --link-color: #008dc0;\n",
       "    --link-color-pale: #97e1fb;\n",
       "    --link-hover: #00688e;\n",
       "}\n",
       "\n",
       "table.dataframe {\n",
       "    font-family: \"Helvetica Neue\", Helvetica, Arial, sans-serif;\n",
       "    font-size: 12px;\n",
       "    background-color: var(--background);\n",
       "    color: var(--text-color-dark);\n",
       "    border: none;\n",
       "    border-collapse: collapse;\n",
       "}\n",
       "\n",
       "table.dataframe th, td {\n",
       "    padding: 6px;\n",
       "    border: 1px solid transparent;\n",
       "    text-align: left;\n",
       "}\n",
       "\n",
       "table.dataframe th {\n",
       "    background-color: var(--background);\n",
       "    color: var(--header-text-color);\n",
       "}\n",
       "\n",
       "table.dataframe td {\n",
       "    vertical-align: top;\n",
       "}\n",
       "\n",
       "table.dataframe th.bottomBorder {\n",
       "    border-bottom-color: var(--bold-border-color);\n",
       "}\n",
       "\n",
       "table.dataframe tbody > tr:nth-child(odd) {\n",
       "    background: var(--background-odd);\n",
       "}\n",
       "\n",
       "table.dataframe tbody > tr:nth-child(even) {\n",
       "    background: var(--background);\n",
       "}\n",
       "\n",
       "table.dataframe tbody > tr:hover {\n",
       "    background: var(--background-hover);\n",
       "}\n",
       "\n",
       "table.dataframe a {\n",
       "    cursor: pointer;\n",
       "    color: var(--link-color);\n",
       "    text-decoration: none;\n",
       "}\n",
       "\n",
       "table.dataframe tr:hover > td a {\n",
       "    color: var(--link-color-pale);\n",
       "}\n",
       "\n",
       "table.dataframe a:hover {\n",
       "    color: var(--link-hover);\n",
       "    text-decoration: underline;\n",
       "}\n",
       "\n",
       "table.dataframe img {\n",
       "    max-width: fit-content;\n",
       "}\n",
       "\n",
       "table.dataframe th.complex {\n",
       "    background-color: var(--background);\n",
       "    border: 1px solid var(--background);\n",
       "}\n",
       "\n",
       "table.dataframe .leftBorder {\n",
       "    border-left-color: var(--inner-border-color);\n",
       "}\n",
       "\n",
       "table.dataframe .rightBorder {\n",
       "    border-right-color: var(--inner-border-color);\n",
       "}\n",
       "\n",
       "table.dataframe .rightAlign {\n",
       "    text-align: right;\n",
       "}\n",
       "\n",
       "table.dataframe .expanderSvg {\n",
       "    width: 8px;\n",
       "    height: 8px;\n",
       "    margin-right: 3px;\n",
       "}\n",
       "\n",
       "table.dataframe .expander {\n",
       "    display: flex;\n",
       "    align-items: center;\n",
       "}\n",
       "\n",
       "/* formatting */\n",
       "\n",
       "table.dataframe .null {\n",
       "    color: var(--text-color-pale);\n",
       "}\n",
       "\n",
       "table.dataframe .structural {\n",
       "    color: var(--text-color-medium);\n",
       "    font-weight: bold;\n",
       "}\n",
       "\n",
       "table.dataframe .dataFrameCaption {\n",
       "    font-weight: bold;\n",
       "}\n",
       "\n",
       "table.dataframe .numbers {\n",
       "    color: var(--text-color-dark);\n",
       "}\n",
       "\n",
       "table.dataframe td:hover .formatted .structural, .null {\n",
       "    color: var(--text-color-dark);\n",
       "}\n",
       "\n",
       "table.dataframe tr:hover .formatted .structural, .null {\n",
       "    color: var(--text-color-dark);\n",
       "}\n",
       "\n",
       "\n",
       "                </style>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%useLatestDescriptors\n",
    "%use dataframe, kotlin-dl"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Read the dataframe from CSV and print the first few lines of it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "        <html>\n",
       "        <head>\n",
       "            <style type=\"text/css\">\n",
       "                \n",
       "\n",
       "            </style>\n",
       "        </head>\n",
       "        <body>\n",
       "            <table class=\"dataframe\" id=\"df_1895825408\"></table>\n",
       "\n",
       "<p>DataFrame: rowsCount = 5, columnsCount = 12</p>\n",
       "        </body>\n",
       "        <script>\n",
       "            /*<!--*/\n",
       "call_DataFrame(function() { DataFrame.addTable({ cols: [{ name: \"<span title=\\\"fixed acidity: Double\\\">fixed acidity</span>\", children: [], rightAlign: true, values: [\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">7.4</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">7.8</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">7.8</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">11.2</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">7.4</span></span>\"] }, \n",
       "{ name: \"<span title=\\\"volatile acidity: Double\\\">volatile acidity</span>\", children: [], rightAlign: true, values: [\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.70</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.88</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.76</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.28</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.70</span></span>\"] }, \n",
       "{ name: \"<span title=\\\"citric acid: Double\\\">citric acid</span>\", children: [], rightAlign: true, values: [\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.00</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.00</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.04</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.56</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.00</span></span>\"] }, \n",
       "{ name: \"<span title=\\\"residual sugar: Double\\\">residual sugar</span>\", children: [], rightAlign: true, values: [\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">1.9</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">2.6</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">2.3</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">1.9</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">1.9</span></span>\"] }, \n",
       "{ name: \"<span title=\\\"chlorides: Double\\\">chlorides</span>\", children: [], rightAlign: true, values: [\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.076</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.098</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.092</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.075</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.076</span></span>\"] }, \n",
       "{ name: \"<span title=\\\"free sulfur dioxide: Double\\\">free sulfur dioxide</span>\", children: [], rightAlign: true, values: [\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">11.0</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">25.0</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">15.0</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">17.0</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">11.0</span></span>\"] }, \n",
       "{ name: \"<span title=\\\"total sulfur dioxide: Double\\\">total sulfur dioxide</span>\", children: [], rightAlign: true, values: [\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">34.0</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">67.0</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">54.0</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">60.0</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">34.0</span></span>\"] }, \n",
       "{ name: \"<span title=\\\"density: Double\\\">density</span>\", children: [], rightAlign: true, values: [\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.9978</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.9968</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.9970</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.9980</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.9978</span></span>\"] }, \n",
       "{ name: \"<span title=\\\"pH: Double\\\">pH</span>\", children: [], rightAlign: true, values: [\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">3.51</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">3.20</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">3.26</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">3.16</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">3.51</span></span>\"] }, \n",
       "{ name: \"<span title=\\\"sulphates: Double\\\">sulphates</span>\", children: [], rightAlign: true, values: [\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.56</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.68</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.65</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.58</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.56</span></span>\"] }, \n",
       "{ name: \"<span title=\\\"alcohol: Double\\\">alcohol</span>\", children: [], rightAlign: true, values: [\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">9.4</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">9.8</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">9.8</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">9.8</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">9.4</span></span>\"] }, \n",
       "{ name: \"<span title=\\\"quality: Int\\\">quality</span>\", children: [], rightAlign: true, values: [\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">5</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">5</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">5</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">6</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">5</span></span>\"] }, \n",
       "], id: 1895825408, rootId: 1895825408, totalRows: 5 } ) });\n",
       "/*-->*/\n",
       "\n",
       "call_DataFrame(function() { DataFrame.renderTable(1895825408) });\n",
       "\n",
       "\n",
       "        </script>\n",
       "        </html>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val raw_df = DataFrame.readCSV(fileOrUrl = \"winequality-red.csv\", delimiter = ';')\n",
    "raw_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "        <html>\n",
       "        <head>\n",
       "            <style type=\"text/css\">\n",
       "                \n",
       "\n",
       "            </style>\n",
       "        </head>\n",
       "        <body>\n",
       "            <table class=\"dataframe\" id=\"df_1895825409\"></table>\n",
       "\n",
       "<p>DataFrame: rowsCount = 12, columnsCount = 13</p>\n",
       "        </body>\n",
       "        <script>\n",
       "            /*<!--*/\n",
       "call_DataFrame(function() { DataFrame.addTable({ cols: [{ name: \"<span title=\\\"column: String\\\">column</span>\", children: [], rightAlign: false, values: [\"fixed acidity\",\"volatile acidity\",\"citric acid\",\"residual sugar\",\"chlorides\",\"free sulfur dioxide\",\"total sulfur dioxide\",\"density\",\"pH\",\"sulphates\",\"alcohol\",\"quality\"] }, \n",
       "{ name: \"<span title=\\\"fixed acidity: Double\\\">fixed acidity</span>\", children: [], rightAlign: true, values: [{ style: \"background-color:#00ff00\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">1.000000</span></span>\"},{ style: \"background-color:#a05e00\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">-0.256131</span></span>\"},{ style: \"background-color:#29d500\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.671703</span></span>\"},{ style: \"background-color:#708e00\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.114777</span></span>\"},{ style: \"background-color:#738b00\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.093705</span></span>\"},{ style: \"background-color:#936b00\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">-0.153794</span></span>\"},{ style: \"background-color:#8d7100\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">-0.113181</span></span>\"},{ style: \"background-color:#2ad400\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.668047</span></span>\"},{ style: \"background-color:#d62800\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">-0.682978</span></span>\"},{ style: \"background-color:#689600\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.183006</span></span>\"},{ style: \"background-color:#877700\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">-0.061668</span></span>\"},{ style: \"background-color:#6f8f00\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.124052</span></span>\"}] }, \n",
       "{ name: \"<span title=\\\"volatile acidity: Double\\\">volatile acidity</span>\", children: [], rightAlign: true, values: [{ style: \"background-color:#a05e00\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">-0.256131</span></span>\"},{ style: \"background-color:#00ff00\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">1.000000</span></span>\"},{ style: \"background-color:#c53900\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">-0.552496</span></span>\"},{ style: \"background-color:#7f7f00\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.001918</span></span>\"},{ style: \"background-color:#778700\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.061298</span></span>\"},{ style: \"background-color:#807e00\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">-0.010504</span></span>\"},{ style: \"background-color:#758900\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.076470</span></span>\"},{ style: \"background-color:#7c8200\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.022026</span></span>\"},{ style: \"background-color:#619d00\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.234937</span></span>\"},{ style: \"background-color:#a05e00\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">-0.260987</span></span>\"},{ style: \"background-color:#996500\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">-0.202288</span></span>\"},{ style: \"background-color:#b14d00\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">-0.390558</span></span>\"}] }, \n",
       "{ name: \"<span title=\\\"citric acid: Double\\\">citric acid</span>\", children: [], rightAlign: true, values: [{ style: \"background-color:#29d500\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.671703</span></span>\"},{ style: \"background-color:#c53900\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">-0.552496</span></span>\"},{ style: \"background-color:#00ff00\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">1.000000</span></span>\"},{ style: \"background-color:#6d9100\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.143577</span></span>\"},{ style: \"background-color:#659900\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.203823</span></span>\"},{ style: \"background-color:#877700\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">-0.060978</span></span>\"},{ style: \"background-color:#7a8400\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.035533</span></span>\"},{ style: \"background-color:#50ae00\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.364947</span></span>\"},{ style: \"background-color:#c43a00\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">-0.541904</span></span>\"},{ style: \"background-color:#57a700\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.312770</span></span>\"},{ style: \"background-color:#718d00\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.109903</span></span>\"},{ style: \"background-color:#629c00\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.226373</span></span>\"}] }, \n",
       "{ name: \"<span title=\\\"residual sugar: Double\\\">residual sugar</span>\", children: [], rightAlign: true, values: [{ style: \"background-color:#708e00\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.114777</span></span>\"},{ style: \"background-color:#7f7f00\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.001918</span></span>\"},{ style: \"background-color:#6d9100\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.143577</span></span>\"},{ style: \"background-color:#00ff00\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">1.000000</span></span>\"},{ style: \"background-color:#788600\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.055610</span></span>\"},{ style: \"background-color:#679700\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.187049</span></span>\"},{ style: \"background-color:#659900\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.203028</span></span>\"},{ style: \"background-color:#52ac00\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.355283</span></span>\"},{ style: \"background-color:#8a7400\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">-0.085652</span></span>\"},{ style: \"background-color:#7e8000\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.005527</span></span>\"},{ style: \"background-color:#7a8400\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.042075</span></span>\"},{ style: \"background-color:#7d8100\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.013732</span></span>\"}] }, \n",
       "{ name: \"<span title=\\\"chlorides: Double\\\">chlorides</span>\", children: [], rightAlign: true, values: [{ style: \"background-color:#738b00\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.093705</span></span>\"},{ style: \"background-color:#778700\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.061298</span></span>\"},{ style: \"background-color:#659900\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.203823</span></span>\"},{ style: \"background-color:#788600\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.055610</span></span>\"},{ style: \"background-color:#00ff00\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">1.000000</span></span>\"},{ style: \"background-color:#7e8000\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.005562</span></span>\"},{ style: \"background-color:#798500\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.047400</span></span>\"},{ style: \"background-color:#659900\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.200632</span></span>\"},{ style: \"background-color:#a15d00\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">-0.265026</span></span>\"},{ style: \"background-color:#50ae00\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.371260</span></span>\"},{ style: \"background-color:#9b6300\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">-0.221141</span></span>\"},{ style: \"background-color:#8f6f00\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">-0.128907</span></span>\"}] }, \n",
       "{ name: \"<span title=\\\"free sulfur dioxide: Double\\\">free sulfur dioxide</span>\", children: [], rightAlign: true, values: [{ style: \"background-color:#936b00\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">-0.153794</span></span>\"},{ style: \"background-color:#807e00\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">-0.010504</span></span>\"},{ style: \"background-color:#877700\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">-0.060978</span></span>\"},{ style: \"background-color:#679700\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.187049</span></span>\"},{ style: \"background-color:#7e8000\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.005562</span></span>\"},{ style: \"background-color:#00ff00\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">1.000000</span></span>\"},{ style: \"background-color:#2ad400\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.667666</span></span>\"},{ style: \"background-color:#827c00\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">-0.021946</span></span>\"},{ style: \"background-color:#768800\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.070377</span></span>\"},{ style: \"background-color:#788600\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.051658</span></span>\"},{ style: \"background-color:#887600\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">-0.069408</span></span>\"},{ style: \"background-color:#857900\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">-0.050656</span></span>\"}] }, \n",
       "{ name: \"<span title=\\\"total sulfur dioxide: Double\\\">total sulfur dioxide</span>\", children: [], rightAlign: true, values: [{ style: \"background-color:#8d7100\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">-0.113181</span></span>\"},{ style: \"background-color:#758900\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.076470</span></span>\"},{ style: \"background-color:#7a8400\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.035533</span></span>\"},{ style: \"background-color:#659900\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.203028</span></span>\"},{ style: \"background-color:#798500\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.047400</span></span>\"},{ style: \"background-color:#2ad400\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.667666</span></span>\"},{ style: \"background-color:#00ff00\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">1.000000</span></span>\"},{ style: \"background-color:#768800\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.071269</span></span>\"},{ style: \"background-color:#877700\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">-0.066495</span></span>\"},{ style: \"background-color:#7a8400\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.042947</span></span>\"},{ style: \"background-color:#996500\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">-0.205654</span></span>\"},{ style: \"background-color:#976700\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">-0.185100</span></span>\"}] }, \n",
       "{ name: \"<span title=\\\"density: Double\\\">density</span>\", children: [], rightAlign: true, values: [{ style: \"background-color:#2ad400\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.668047</span></span>\"},{ style: \"background-color:#7c8200\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.022026</span></span>\"},{ style: \"background-color:#50ae00\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.364947</span></span>\"},{ style: \"background-color:#52ac00\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.355283</span></span>\"},{ style: \"background-color:#659900\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.200632</span></span>\"},{ style: \"background-color:#827c00\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">-0.021946</span></span>\"},{ style: \"background-color:#768800\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.071269</span></span>\"},{ style: \"background-color:#00ff00\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">1.000000</span></span>\"},{ style: \"background-color:#ab5300\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">-0.341699</span></span>\"},{ style: \"background-color:#6c9200\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.148506</span></span>\"},{ style: \"background-color:#be4000\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">-0.496180</span></span>\"},{ style: \"background-color:#956900\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">-0.174919</span></span>\"}] }, \n",
       "{ name: \"<span title=\\\"pH: Double\\\">pH</span>\", children: [], rightAlign: true, values: [{ style: \"background-color:#d62800\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">-0.682978</span></span>\"},{ style: \"background-color:#619d00\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.234937</span></span>\"},{ style: \"background-color:#c43a00\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">-0.541904</span></span>\"},{ style: \"background-color:#8a7400\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">-0.085652</span></span>\"},{ style: \"background-color:#a15d00\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">-0.265026</span></span>\"},{ style: \"background-color:#768800\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.070377</span></span>\"},{ style: \"background-color:#877700\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">-0.066495</span></span>\"},{ style: \"background-color:#ab5300\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">-0.341699</span></span>\"},{ style: \"background-color:#00ff00\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">1.000000</span></span>\"},{ style: \"background-color:#986600\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">-0.196648</span></span>\"},{ style: \"background-color:#659900\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.205633</span></span>\"},{ style: \"background-color:#867800\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">-0.057731</span></span>\"}] }, \n",
       "{ name: \"<span title=\\\"sulphates: Double\\\">sulphates</span>\", children: [], rightAlign: true, values: [{ style: \"background-color:#689600\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.183006</span></span>\"},{ style: \"background-color:#a05e00\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">-0.260987</span></span>\"},{ style: \"background-color:#57a700\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.312770</span></span>\"},{ style: \"background-color:#7e8000\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.005527</span></span>\"},{ style: \"background-color:#50ae00\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.371260</span></span>\"},{ style: \"background-color:#788600\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.051658</span></span>\"},{ style: \"background-color:#7a8400\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.042947</span></span>\"},{ style: \"background-color:#6c9200\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.148506</span></span>\"},{ style: \"background-color:#986600\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">-0.196648</span></span>\"},{ style: \"background-color:#00ff00\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">1.000000</span></span>\"},{ style: \"background-color:#738b00\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.093595</span></span>\"},{ style: \"background-color:#5f9f00\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.251397</span></span>\"}] }, \n",
       "{ name: \"<span title=\\\"alcohol: Double\\\">alcohol</span>\", children: [], rightAlign: true, values: [{ style: \"background-color:#877700\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">-0.061668</span></span>\"},{ style: \"background-color:#996500\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">-0.202288</span></span>\"},{ style: \"background-color:#718d00\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.109903</span></span>\"},{ style: \"background-color:#7a8400\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.042075</span></span>\"},{ style: \"background-color:#9b6300\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">-0.221141</span></span>\"},{ style: \"background-color:#887600\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">-0.069408</span></span>\"},{ style: \"background-color:#996500\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">-0.205654</span></span>\"},{ style: \"background-color:#be4000\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">-0.496180</span></span>\"},{ style: \"background-color:#659900\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.205633</span></span>\"},{ style: \"background-color:#738b00\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.093595</span></span>\"},{ style: \"background-color:#00ff00\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">1.000000</span></span>\"},{ style: \"background-color:#42bc00\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.476166</span></span>\"}] }, \n",
       "{ name: \"<span title=\\\"quality: Double\\\">quality</span>\", children: [], rightAlign: true, values: [{ style: \"background-color:#6f8f00\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.124052</span></span>\"},{ style: \"background-color:#b14d00\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">-0.390558</span></span>\"},{ style: \"background-color:#629c00\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.226373</span></span>\"},{ style: \"background-color:#7d8100\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.013732</span></span>\"},{ style: \"background-color:#8f6f00\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">-0.128907</span></span>\"},{ style: \"background-color:#857900\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">-0.050656</span></span>\"},{ style: \"background-color:#976700\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">-0.185100</span></span>\"},{ style: \"background-color:#956900\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">-0.174919</span></span>\"},{ style: \"background-color:#867800\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">-0.057731</span></span>\"},{ style: \"background-color:#5f9f00\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.251397</span></span>\"},{ style: \"background-color:#42bc00\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.476166</span></span>\"},{ style: \"background-color:#00ff00\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">1.000000</span></span>\"}] }, \n",
       "], id: 1895825409, rootId: 1895825409, totalRows: 12 } ) });\n",
       "/*-->*/\n",
       "\n",
       "call_DataFrame(function() { DataFrame.renderTable(1895825409) });\n",
       "\n",
       "\n",
       "        </script>\n",
       "        </html>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw_df.corr().format { colsOf<Double>() }.with { linearBg(it, -1.0 to red, 1.0 to green) }"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Based on the correlation, we can remove some columns, they seem to be insignificant"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "val df = raw_df.remove {`free sulfur dioxide` and `residual sugar` and pH }"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Predict wine quality: first approach"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "// Simple converter function between DataFrame and KotlinDL data representations\n",
    "fun <T> DataFrame<T>.toOnHeapDataset(labelColumnName: String): OnHeapDataset {\n",
    "    return OnHeapDataset.create(\n",
    "        dataframe = this,\n",
    "        yColumn = labelColumnName\n",
    "    )\n",
    "}\n",
    "\n",
    "fun OnHeapDataset.Companion.create(\n",
    "    dataframe: DataFrame<Any?>,\n",
    "    yColumn: String\n",
    "): OnHeapDataset {\n",
    "    fun extractX(): Array<FloatArray> =\n",
    "        dataframe.remove(yColumn).rows()\n",
    "            .map { (it.values() as List<Float>).toFloatArray() }.toTypedArray()\n",
    "\n",
    "    fun extractY(): FloatArray =\n",
    "        dataframe.get { yColumn<Float>() }.toList().toFloatArray()\n",
    "\n",
    "    return create(\n",
    "        ::extractX,\n",
    "        ::extractY\n",
    "    )\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "val (train, test) = df.convert { colsOf<Double>() }.toFloat()\n",
    "        .toOnHeapDataset(labelColumnName = \"quality\")\n",
    "        .split(0.8)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define simple neural network with only 2 dense layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "val inputNeurons = train.x[0].size.toLong()\n",
    "\n",
    "val model = Sequential.of(\n",
    "    Input(\n",
    "        inputNeurons\n",
    "    ),\n",
    "    Dense(\n",
    "        outputSize = (inputNeurons * 10).toInt(),\n",
    "        activation = Activations.Tanh,\n",
    "        kernelInitializer = HeNormal(),\n",
    "        biasInitializer = HeNormal()\n",
    "    ),\n",
    "    Dense(\n",
    "        outputSize = (inputNeurons * 10).toInt(),\n",
    "        activation = Activations.Tanh,\n",
    "        kernelInitializer = HeNormal(),\n",
    "        biasInitializer = HeNormal()\n",
    "    ),\n",
    "    Dense(\n",
    "        outputSize = 1,\n",
    "        activation = Activations.Linear,\n",
    "        kernelInitializer = HeNormal(),\n",
    "        biasInitializer = HeNormal()\n",
    "    )\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(optimizer = Adam(), loss = Losses.MSE, metric = Metrics.MAE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==============================================================================\n",
      "Model type: Sequential\n",
      "______________________________________________________________________________\n",
      "Layer (type)                           Output Shape              Param #      \n",
      "==============================================================================\n",
      "input_1(Input)                         [None, 8]                 0            \n",
      "______________________________________________________________________________\n",
      "dense_2(Dense)                         [None, 80]                720          \n",
      "______________________________________________________________________________\n",
      "dense_3(Dense)                         [None, 80]                6480         \n",
      "______________________________________________________________________________\n",
      "dense_4(Dense)                         [None, 1]                 81           \n",
      "______________________________________________________________________________\n",
      "==============================================================================\n",
      "Total trainable params: 7281\n",
      "Total frozen params: 0\n",
      "Total params: 7281\n",
      "==============================================================================\n"
     ]
    }
   ],
   "source": [
    "model.printSummary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Train it!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "val trainHist = model.fit(train, batchSize = 500, epochs=2000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "        <html>\n",
       "        <head>\n",
       "            <style type=\"text/css\">\n",
       "                \n",
       "\n",
       "            </style>\n",
       "        </head>\n",
       "        <body>\n",
       "            <table class=\"dataframe\" id=\"df_1895825410\"></table>\n",
       "\n",
       "<p>DataFrame: rowsCount = 5, columnsCount = 5</p>\n",
       "        </body>\n",
       "        <script>\n",
       "            /*<!--*/\n",
       "call_DataFrame(function() { DataFrame.addTable({ cols: [{ name: \"<span title=\\\"epochIndex: Int\\\">epochIndex</span>\", children: [], rightAlign: true, values: [\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">1996</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">1997</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">1998</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">1999</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">2000</span></span>\"] }, \n",
       "{ name: \"<span title=\\\"lossValue: Double\\\">lossValue</span>\", children: [], rightAlign: true, values: [\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.334507</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.334469</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.334430</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.334392</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.334353</span></span>\"] }, \n",
       "{ name: \"<span title=\\\"metricValues: List<Double>\\\">metricValues</span>\", children: [], rightAlign: false, values: [\"<span class=\\\"formatted\\\" title=\\\"0.4508481025695801\\\"><span class=\\\"structural\\\">[</span><span class=\\\"numbers\\\">0.5</span><span class=\\\"structural\\\">]</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"0.45081400871276855\\\"><span class=\\\"structural\\\">[</span><span class=\\\"numbers\\\">0.5</span><span class=\\\"structural\\\">]</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"0.45078006386756897\\\"><span class=\\\"structural\\\">[</span><span class=\\\"numbers\\\">0.5</span><span class=\\\"structural\\\">]</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"0.4507458209991455\\\"><span class=\\\"structural\\\">[</span><span class=\\\"numbers\\\">0.5</span><span class=\\\"structural\\\">]</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"0.4507113993167877\\\"><span class=\\\"structural\\\">[</span><span class=\\\"numbers\\\">0.5</span><span class=\\\"structural\\\">]</span></span>\"] }, \n",
       "{ name: \"<span title=\\\"valLossValue: Double\\\">valLossValue</span>\", children: [], rightAlign: true, values: [\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">NaN</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">NaN</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">NaN</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">NaN</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">NaN</span></span>\"] }, \n",
       "{ name: \"<span title=\\\"valMetricValues: List<Double?>\\\">valMetricValues</span>\", children: [], rightAlign: false, values: [\"<span class=\\\"formatted\\\" title=\\\"NaN\\\"><span class=\\\"structural\\\">[</span><span class=\\\"numbers\\\">NaN</span><span class=\\\"structural\\\">]</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"NaN\\\"><span class=\\\"structural\\\">[</span><span class=\\\"numbers\\\">NaN</span><span class=\\\"structural\\\">]</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"NaN\\\"><span class=\\\"structural\\\">[</span><span class=\\\"numbers\\\">NaN</span><span class=\\\"structural\\\">]</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"NaN\\\"><span class=\\\"structural\\\">[</span><span class=\\\"numbers\\\">NaN</span><span class=\\\"structural\\\">]</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"NaN\\\"><span class=\\\"structural\\\">[</span><span class=\\\"numbers\\\">NaN</span><span class=\\\"structural\\\">]</span></span>\"] }, \n",
       "], id: 1895825410, rootId: 1895825410, totalRows: 5 } ) });\n",
       "/*-->*/\n",
       "\n",
       "call_DataFrame(function() { DataFrame.renderTable(1895825410) });\n",
       "\n",
       "\n",
       "        </script>\n",
       "        </html>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainHist.epochHistory.toDataFrame().tail()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's check that our network predicts values more or less correctly:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5.24826"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.predictSoftly(test.x[9])[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5.0"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.y[9]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Close the model:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Predict wine quality: second approach"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "fun <T> trainTestSplit(d: DataFrame<T>, col: String, trainPart: Double): Pair<Pair<DataFrame<T>, DataFrame<T>>, Pair<DataFrame<T>, DataFrame<T>>> {\n",
    "    val n = d.count()\n",
    "    val trainN = ceil(n * trainPart).toInt()\n",
    "\n",
    "    val shuffledInd = (0 until n).shuffled()\n",
    "    val trainInd = shuffledInd.subList(0, trainN)\n",
    "    val testInd = shuffledInd.subList(trainN, n)\n",
    "    \n",
    "    val train = d[trainInd]\n",
    "    val test = d[testInd]\n",
    "    \n",
    "    val trainX = train.select { all().except(cols(col)) }\n",
    "    val trainY = train.select(col)\n",
    "    \n",
    "    val testX = test.select { all().except(cols(col)) }\n",
    "    val testY = test.select(col)\n",
    "    \n",
    "    return (trainX to trainY) to (testX to testY)\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's create and then train the model as we did before"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "val (trainPair, testPair) = trainTestSplit(df, \"quality\", 0.8)\n",
    "val (trainX, trainY) = trainPair\n",
    "val (testX, testY) = testPair"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "fun <T> DataFrame<T>.toX(): Array<FloatArray> = \n",
    "        merge { colsOf<Number>() }.by { it.map { it.toFloat() }.toFloatArray() }.into(\"X\")\n",
    "        .get { \"X\"<FloatArray>() }.toList().toTypedArray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "fun <T> DataFrame<T>.toY() = get { \"quality\"<Int>() }.asIterable().map { it.toFloat() }.toFloatArray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "val trainXDL = trainX.toX()\n",
    "val trainYDL = trainY.toY()\n",
    "val testXDL = testX.toX()\n",
    "val testYDL = testY.toY()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "val trainKotlinDLDataset = OnHeapDataset.create({trainXDL}, {trainYDL})\n",
    "val testKotlinDLDataset = OnHeapDataset.create({testXDL}, {testYDL})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==============================================================================\n",
      "Model type: Sequential\n",
      "______________________________________________________________________________\n",
      "Layer (type)                           Output Shape              Param #      \n",
      "==============================================================================\n",
      "input_1(Input)                         [None, 8]                 0            \n",
      "______________________________________________________________________________\n",
      "dense_2(Dense)                         [None, 80]                720          \n",
      "______________________________________________________________________________\n",
      "dense_3(Dense)                         [None, 80]                6480         \n",
      "______________________________________________________________________________\n",
      "dense_4(Dense)                         [None, 1]                 81           \n",
      "______________________________________________________________________________\n",
      "==============================================================================\n",
      "Total trainable params: 7281\n",
      "Total frozen params: 0\n",
      "Total params: 7281\n",
      "==============================================================================\n"
     ]
    }
   ],
   "source": [
    "val inputNeurons = train.x[0].size.toLong()\n",
    "\n",
    "val model2 = Sequential.of(\n",
    "    Input(\n",
    "        inputNeurons\n",
    "    ),\n",
    "    Dense(\n",
    "        outputSize = (inputNeurons * 10).toInt(),\n",
    "        activation = Activations.Tanh,\n",
    "        kernelInitializer = HeNormal(),\n",
    "        biasInitializer = HeNormal()\n",
    "    ),\n",
    "    Dense(\n",
    "        outputSize = (inputNeurons * 10).toInt(),\n",
    "        activation = Activations.Tanh,\n",
    "        kernelInitializer = HeNormal(),\n",
    "        biasInitializer = HeNormal()\n",
    "    ),\n",
    "    Dense(\n",
    "        outputSize = 1,\n",
    "        activation = Activations.Linear,\n",
    "        kernelInitializer = HeNormal(),\n",
    "        biasInitializer = HeNormal()\n",
    "    )\n",
    ")\n",
    "model2.compile(optimizer = Adam(), loss = Losses.MSE, metric = Metrics.MAE)\n",
    "model2.printSummary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "        <html>\n",
       "        <head>\n",
       "            <style type=\"text/css\">\n",
       "                \n",
       "\n",
       "            </style>\n",
       "        </head>\n",
       "        <body>\n",
       "            <table class=\"dataframe\" id=\"df_1895825411\"></table>\n",
       "\n",
       "<p>DataFrame: rowsCount = 5, columnsCount = 5</p>\n",
       "        </body>\n",
       "        <script>\n",
       "            /*<!--*/\n",
       "call_DataFrame(function() { DataFrame.addTable({ cols: [{ name: \"<span title=\\\"epochIndex: Int\\\">epochIndex</span>\", children: [], rightAlign: true, values: [\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">1996</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">1997</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">1998</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">1999</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">2000</span></span>\"] }, \n",
       "{ name: \"<span title=\\\"lossValue: Double\\\">lossValue</span>\", children: [], rightAlign: true, values: [\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.334803</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.334767</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.334730</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.334693</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.334656</span></span>\"] }, \n",
       "{ name: \"<span title=\\\"metricValues: List<Double>\\\">metricValues</span>\", children: [], rightAlign: false, values: [\"<span class=\\\"formatted\\\" title=\\\"0.45111215114593506\\\"><span class=\\\"structural\\\">[</span><span class=\\\"numbers\\\">0.5</span><span class=\\\"structural\\\">]</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"0.4510812759399414\\\"><span class=\\\"structural\\\">[</span><span class=\\\"numbers\\\">0.5</span><span class=\\\"structural\\\">]</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"0.45104971528053284\\\"><span class=\\\"structural\\\">[</span><span class=\\\"numbers\\\">0.5</span><span class=\\\"structural\\\">]</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"0.4510185718536377\\\"><span class=\\\"structural\\\">[</span><span class=\\\"numbers\\\">0.5</span><span class=\\\"structural\\\">]</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"0.45098742842674255\\\"><span class=\\\"structural\\\">[</span><span class=\\\"numbers\\\">0.5</span><span class=\\\"structural\\\">]</span></span>\"] }, \n",
       "{ name: \"<span title=\\\"valLossValue: Double\\\">valLossValue</span>\", children: [], rightAlign: true, values: [\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">NaN</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">NaN</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">NaN</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">NaN</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">NaN</span></span>\"] }, \n",
       "{ name: \"<span title=\\\"valMetricValues: List<Double?>\\\">valMetricValues</span>\", children: [], rightAlign: false, values: [\"<span class=\\\"formatted\\\" title=\\\"NaN\\\"><span class=\\\"structural\\\">[</span><span class=\\\"numbers\\\">NaN</span><span class=\\\"structural\\\">]</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"NaN\\\"><span class=\\\"structural\\\">[</span><span class=\\\"numbers\\\">NaN</span><span class=\\\"structural\\\">]</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"NaN\\\"><span class=\\\"structural\\\">[</span><span class=\\\"numbers\\\">NaN</span><span class=\\\"structural\\\">]</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"NaN\\\"><span class=\\\"structural\\\">[</span><span class=\\\"numbers\\\">NaN</span><span class=\\\"structural\\\">]</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"NaN\\\"><span class=\\\"structural\\\">[</span><span class=\\\"numbers\\\">NaN</span><span class=\\\"structural\\\">]</span></span>\"] }, \n",
       "], id: 1895825411, rootId: 1895825411, totalRows: 5 } ) });\n",
       "/*-->*/\n",
       "\n",
       "call_DataFrame(function() { DataFrame.renderTable(1895825411) });\n",
       "\n",
       "\n",
       "        </script>\n",
       "        </html>"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val trainHist = model2.fit(train, batchSize = 500, epochs=2000)\n",
    "trainHist.epochHistory.toDataFrame().tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5.9178224"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model2.predictSoftly(testXDL[9])[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6.0"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "testYDL[9]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can also compare predicted and ground truth values to ensure predictions are correct"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "val predicted = testXDL.mapIndexed  { i, _ -> \n",
    "        round(model2.predictSoftly(testXDL[i])[0]).toInt()\n",
    "    }.toColumn(\"predicted\")\n",
    "\n",
    "val ground_truth = testYDL.mapIndexed  { i, _ -> \n",
    "       testYDL[i].toInt()\n",
    "    }.toColumn(\"ground_truth\")\n",
    "\n",
    "val predDf = dataFrameOf(predicted, ground_truth)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "        <html>\n",
       "        <head>\n",
       "            <style type=\"text/css\">\n",
       "                \n",
       "\n",
       "            </style>\n",
       "        </head>\n",
       "        <body>\n",
       "            <table class=\"dataframe\" id=\"df_1895825412\"></table>\n",
       "\n",
       "<p>DataFrame: rowsCount = 5, columnsCount = 2</p>\n",
       "        </body>\n",
       "        <script>\n",
       "            /*<!--*/\n",
       "call_DataFrame(function() { DataFrame.addTable({ cols: [{ name: \"<span title=\\\"predicted: Int\\\">predicted</span>\", children: [], rightAlign: true, values: [\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">6</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">5</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">5</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">5</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">6</span></span>\"] }, \n",
       "{ name: \"<span title=\\\"ground_truth: Int\\\">ground_truth</span>\", children: [], rightAlign: true, values: [\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">5</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">6</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">5</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">5</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">6</span></span>\"] }, \n",
       "], id: 1895825412, rootId: 1895825412, totalRows: 5 } ) });\n",
       "/*-->*/\n",
       "\n",
       "call_DataFrame(function() { DataFrame.renderTable(1895825412) });\n",
       "\n",
       "\n",
       "        </script>\n",
       "        </html>"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predDf.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "        <html>\n",
       "        <head>\n",
       "            <style type=\"text/css\">\n",
       "                \n",
       "\n",
       "            </style>\n",
       "        </head>\n",
       "        <body>\n",
       "            <table class=\"dataframe\" id=\"df_1895825413\"></table>\n",
       "\n",
       "<p>DataFrame: rowsCount = 6, columnsCount = 5</p>\n",
       "        </body>\n",
       "        <script>\n",
       "            /*<!--*/\n",
       "call_DataFrame(function() { DataFrame.addTable({ cols: [{ name: \"<span title=\\\"ground_truth: Int\\\">ground_truth</span>\", children: [], rightAlign: true, values: [\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">3</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">4</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">5</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">6</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">7</span></span>\",\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">8</span></span>\"] }, \n",
       "{ name: \"<span title=\\\"6: Int\\\">6</span>\", children: [], rightAlign: true, values: [{ style: \"background-color:#32be32\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0</span></span>\"},{ style: \"background-color:#32d232\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0</span></span>\"},{ style: \"background-color:#32e632\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">37</span></span>\"},{ style: \"background-color:#32fa32\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">89</span></span>\"},{ style: \"background-color:#32e632\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">19</span></span>\"},{ style: \"background-color:#32d232\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0</span></span>\"}] }, \n",
       "{ name: \"<span title=\\\"5: Int\\\">5</span>\", children: [], rightAlign: true, values: [{ style: \"background-color:#32d232\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0</span></span>\"},{ style: \"background-color:#32e632\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">7</span></span>\"},{ style: \"background-color:#32fa32\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">99</span></span>\"},{ style: \"background-color:#32e632\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">38</span></span>\"},{ style: \"background-color:#32d232\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0</span></span>\"},{ style: \"background-color:#32be32\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0</span></span>\"}] }, \n",
       "{ name: \"<span title=\\\"7: Int\\\">7</span>\", children: [], rightAlign: true, values: [{ style: \"background-color:#32aa32\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0</span></span>\"},{ style: \"background-color:#32be32\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0</span></span>\"},{ style: \"background-color:#32d232\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0</span></span>\"},{ style: \"background-color:#32e632\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">7</span></span>\"},{ style: \"background-color:#32fa32\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">18</span></span>\"},{ style: \"background-color:#32e632\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">4</span></span>\"}] }, \n",
       "{ name: \"<span title=\\\"4: Int\\\">4</span>\", children: [], rightAlign: true, values: [{ style: \"background-color:#32e632\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">1</span></span>\"},{ style: \"background-color:#32fa32\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0</span></span>\"},{ style: \"background-color:#32e632\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0</span></span>\"},{ style: \"background-color:#32d232\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0</span></span>\"},{ style: \"background-color:#32be32\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0</span></span>\"},{ style: \"background-color:#32aa32\", value: \"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0</span></span>\"}] }, \n",
       "], id: 1895825413, rootId: 1895825413, totalRows: 6 } ) });\n",
       "/*-->*/\n",
       "\n",
       "call_DataFrame(function() { DataFrame.renderTable(1895825413) });\n",
       "\n",
       "\n",
       "        </script>\n",
       "        </html>"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val inds = List(10){it + 1}\n",
    "val ctab = predDf.groupBy { ground_truth }.pivotCounts(inward = false) { predicted }.sortBy { ground_truth }\n",
    "\n",
    "ctab.format { drop(1) }.perRowCol { row, col ->\n",
    "    val y = col.name().toInt()\n",
    "    val x = row.ground_truth\n",
    "    val k = 1.0 - abs(x - y)/10.0\n",
    "    background(RGBColor(50, (50 + k * 200).toInt().toShort(), 50))\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "val predDf2 = predDf.add(\"avg_dev\") { abs(predicted - ground_truth) }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "        <html>\n",
       "        <head>\n",
       "            <style type=\"text/css\">\n",
       "                \n",
       "\n",
       "            </style>\n",
       "        </head>\n",
       "        <body>\n",
       "            <table class=\"dataframe\" id=\"df_1895825414\"></table>\n",
       "\n",
       "<p>DataFrame: rowsCount = 1, columnsCount = 12</p>\n",
       "        </body>\n",
       "        <script>\n",
       "            /*<!--*/\n",
       "call_DataFrame(function() { DataFrame.addTable({ cols: [{ name: \"<span title=\\\"name: String\\\">name</span>\", children: [], rightAlign: false, values: [\"avg_dev\"] }, \n",
       "{ name: \"<span title=\\\"type: Any\\\">type</span>\", children: [], rightAlign: false, values: [\"Int\"] }, \n",
       "{ name: \"<span title=\\\"count: Int\\\">count</span>\", children: [], rightAlign: true, values: [\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">319</span></span>\"] }, \n",
       "{ name: \"<span title=\\\"unique: Int\\\">unique</span>\", children: [], rightAlign: true, values: [\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">2</span></span>\"] }, \n",
       "{ name: \"<span title=\\\"nulls: Int\\\">nulls</span>\", children: [], rightAlign: true, values: [\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0</span></span>\"] }, \n",
       "{ name: \"<span title=\\\"top: Int\\\">top</span>\", children: [], rightAlign: true, values: [\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0</span></span>\"] }, \n",
       "{ name: \"<span title=\\\"freq: Int\\\">freq</span>\", children: [], rightAlign: true, values: [\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">206</span></span>\"] }, \n",
       "{ name: \"<span title=\\\"mean: Double\\\">mean</span>\", children: [], rightAlign: true, values: [\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.354232</span></span>\"] }, \n",
       "{ name: \"<span title=\\\"std: Double\\\">std</span>\", children: [], rightAlign: true, values: [\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0.479031</span></span>\"] }, \n",
       "{ name: \"<span title=\\\"min: Int\\\">min</span>\", children: [], rightAlign: true, values: [\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0</span></span>\"] }, \n",
       "{ name: \"<span title=\\\"median: Int\\\">median</span>\", children: [], rightAlign: true, values: [\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">0</span></span>\"] }, \n",
       "{ name: \"<span title=\\\"max: Int\\\">max</span>\", children: [], rightAlign: true, values: [\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">1</span></span>\"] }, \n",
       "], id: 1895825414, rootId: 1895825414, totalRows: 1 } ) });\n",
       "/*-->*/\n",
       "\n",
       "call_DataFrame(function() { DataFrame.renderTable(1895825414) });\n",
       "\n",
       "\n",
       "        </script>\n",
       "        </html>"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predDf2.avg_dev.cast<Double>().describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "        <html>\n",
       "        <head>\n",
       "            <style type=\"text/css\">\n",
       "                \n",
       "\n",
       "            </style>\n",
       "        </head>\n",
       "        <body>\n",
       "            <table class=\"dataframe\" id=\"df_1895825415\"></table>\n",
       "\n",
       "<p>DataRow: index = 222, columnsCount = 3</p>\n",
       "        </body>\n",
       "        <script>\n",
       "            /*<!--*/\n",
       "call_DataFrame(function() { DataFrame.addTable({ cols: [{ name: \"<span title=\\\"predicted: Int\\\">predicted</span>\", children: [], rightAlign: true, values: [\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">6</span></span>\"] }, \n",
       "{ name: \"<span title=\\\"ground_truth: Int\\\">ground_truth</span>\", children: [], rightAlign: true, values: [\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">5</span></span>\"] }, \n",
       "{ name: \"<span title=\\\"avg_dev: Int\\\">avg_dev</span>\", children: [], rightAlign: true, values: [\"<span class=\\\"formatted\\\" title=\\\"\\\"><span class=\\\"numbers\\\">1</span></span>\"] }, \n",
       "], id: 1895825415, rootId: 1895825415, totalRows: 1 } ) });\n",
       "/*-->*/\n",
       "\n",
       "call_DataFrame(function() { DataFrame.renderTable(1895825415) });\n",
       "\n",
       "\n",
       "        </script>\n",
       "        </html>"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predDf2.sortBy { avg_dev }[(0.7 * (319 - 1)).toInt()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "model2.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Kotlin",
   "language": "kotlin",
   "name": "kotlin"
  },
  "language_info": {
   "codemirror_mode": "text/x-kotlin",
   "file_extension": ".kt",
   "mimetype": "text/x-kotlin",
   "name": "kotlin",
   "nbconvert_exporter": "",
   "pygments_lexer": "kotlin",
   "version": "1.8.0-dev-707"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
